/*
 * Copyright (c) 2017. Lorem ipsum dolor sit amet, consectetur adipiscing elit.
 * Morbi non lorem porttitor neque feugiat blandit. Ut vitae ipsum eget quam lacinia accumsan.
 * Etiam sed turpis ac ipsum condimentum fringilla. Maecenas magna.
 * Proin dapibus sapien vel ante. Aliquam erat volutpat. Pellentesque sagittis ligula eget metus.
 * Vestibulum commodo. Ut rhoncus gravida arcu.
 */

package hello.jsoup;

import com.alibaba.fastjson.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.awt.image.BufferedImage;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;

/**
 * @Author hiberk
 * @Date 2017/8/1 16:13
 * @Description this is used for crawling some pictures from a specific website
 * 在资源路径中放crawl.properties 下载Jsoup依赖
 * #图片的存放路径
 * picture.path = F:\\
 * #图片地址备份
 * text.path = F:\\post.txt
 * @see <a href="http://m.7kk.com">美女</a>
 * @use Jsoup
 * @version 1.7.2
 */
public class CrawlBeauty{

    public static ArrayList<String> urlList = new ArrayList<String>();

    public static Set<String> jpgUrl = new HashSet<String>();

    public static ArrayList<String> jpgName = new ArrayList<String>();

    public void setPicturePath(String picturePath) {
        this.picturePath = picturePath;
    }

    public void setTextPath(String textPath) {
        this.textPath = textPath;
    }

    private String picturePath;
    private String textPath;
    private String targetHtml;
    private String destHtml;

    public void setTargetHtml(String targetHtml) {
        this.targetHtml = targetHtml;
    }

    public void setDestHtml(String destHtml) {
        this.destHtml = destHtml;
    }

    public static void main(String[] args) {
        CrawlBeauty crawlBeauty = new CrawlBeauty();
        crawlBeauty.start();
    }
    //启动入口
    public void start() {
        InputStream in = null;
        try {
            in = CrawlBeauty.class.getClassLoader().getResourceAsStream("crawl.properties");
            Properties p = new Properties();
            p.load(in);
            setPicturePath(p.getProperty("picture.path"));
            setTextPath(p.getProperty("text.path"));
            setTargetHtml(p.getProperty("target.html"));
            setDestHtml(p.getProperty("dest.html"));
            in.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        spider("index.php?page=1");
    }

    /**
     * ajax的请求返回json数据。直接取出其中的图片
     * @status
     * @html
     * @nextUrl 下一个页面的地址
     * @param nextUrl 当前的请求地址
     * @return nextGoal 返回下一页的请求地址
     */
    public void spider(String nextUrl) {

        Document doc = null;
        Elements results = null;
        Connection conn = null;
        String nextGoal = "";
        try {
            conn = Jsoup.connect("http://m.7kk.com/" + nextUrl).timeout(10000).data("is_ajax", "1")
                    .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0").method(Connection.Method.POST)
                    .ignoreContentType(true);
            Connection.Response response = conn.execute();
            String s = response.body();
            JSONObject b = JSONObject.parseObject(s);
            Object obj = b.get("html");
            nextGoal = b.get("nextUrl").toString();
            doc = Jsoup.parse(obj.toString());
            results = doc.getElementsByTag("img");
            File file = new File(textPath);
            if (!file.exists()) {
                file.createNewFile();
            }
            //向文件的后面追加 不覆盖之前的
            FileOutputStream pw = new FileOutputStream(file, true);
            for (Element result : results)
            {
                //将这些图片的链接存到一个post.txt文件中
                pw.write((result.attr("src") + "\r\n").getBytes());
                //下载这些文件
//                downLoadPicture(result.attr("src"));
            }
            //如果下一个请求地址不为空(即还有下一页内容) 则递归调用自身
            if (!nextGoal.equals("")) {
                spider(nextGoal);
            }
            pw.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            htmlEdit();
            System.out.println("successful!");
        }

    }
    @Test
    public void crawl() {
        Document doc = null;
        Elements results = null;
        Connection conn = null;
        try {
            conn = Jsoup.connect("http://www.zjito.com/").timeout(10000)
                    .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0").method(Connection.Method.GET)
                    .ignoreContentType(true);
            Connection.Response response = conn.execute();
            String s = response.body();
            doc = Jsoup.parse(s);
            //获取所有的大图跳转链接
            results = doc.getElementsByAttributeValue("class", "link");
            for (Element result : results)
            {
                //再次判断是a 标签
                if (result.childNodes().size() == 1) {
                    urlList.add(result.attr("href"));
                }
            }
            //将这个地址清单依次下载
            downLoadJPG(urlList);
            htmlEdit();
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public void downLoadJPG(ArrayList<String> urlList) {
        for (String url : urlList) {
            iterator(url, 1);
        }
    }

    public void iterator(String url, Integer page) {

        Document doc = null;
        Elements results = null;
        Connection conn = null;
            try {
                conn = Jsoup.connect(url + "?idx=" + page).timeout(10000)
                        .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0").method(Connection.Method.GET)
                        .ignoreContentType(true);
                Connection.Response response = null;
                response = conn.execute();
                if (response.statusCode() != 200) {
                    return ;
                }
                String s = response.body();
                doc = Jsoup.parse(s);
                Elements elements = doc.getElementsByClass("div-num");
                for (Element result : elements) {
                    jpgUrl.add(result.attr("data-src"));
                }
                //下载图片
                downLoadPicture(jpgUrl);
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                return ;
            }
    }

    /**
     * 根据请求地址下载相应的文件到本地
     */
    public void downLoadPicture(Set<String>  jpgUrl) {
        URL url = null;
        URLConnection urlconn = null;
        BufferedInputStream br = null;
        BufferedOutputStream  out = null;
        try {
            for (String goalUrl : jpgUrl) {
                url = new URL(goalUrl);
                urlconn = url.openConnection();
                //根据请求路径获得图片的名称
                String fileName = goalUrl.substring(goalUrl.lastIndexOf("/") + 1, goalUrl.lastIndexOf(".")) + ".jpg";
                //写出文件的时候要用字节流
                out = new BufferedOutputStream(new FileOutputStream(
                        new File("C:\\Users\\Administrator\\Desktop\\beauty\\img\\", fileName)));
                InputStream is = urlconn.getInputStream();
                br = new BufferedInputStream(is);
                int i;
                while ((i = br.read()) != -1) {
                    out.write(i);
                }
                out.flush();
                System.out.println("获取成功！");
                jpgName.add(fileName);
            }
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                br.close();
                out.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    /**
     * 将图片路径写入到html文件中
     * @return
     */
    @Test
    public void htmlEdit()
    {
        try
        {
            File file = new File("C:\\Users\\Administrator\\Desktop\\beauty\\welcome.html");
            Document doc = Jsoup.parse(file, "utf-8");
            Element div = doc.getElementById("owl-demo");
            //a标签和img标签
            Elements allNode = div.children();
            //a节点
            Node aLabel = null;
            //图片节点
            Node imgLabel = null;
            for (Element node : allNode) {
                //img标签
                imgLabel = node.childNode(0);
                aLabel = imgLabel.parent();
            }
            File destHtml = new File("C:\\Users\\Administrator\\Desktop\\beauty\\welcome.html");
            if (destHtml.exists()) {
                //先删除后创建一个新的文件
                destHtml.delete();
                destHtml.createNewFile();
            }
            //移除原来的所有节点
            doc.getElementsByClass("item").remove();
            BufferedWriter bw = new BufferedWriter(new FileWriter(destHtml));
            String line;
            String appendHtml = "";
            getFileName();
            for (String fileName : jpgName) {
                imgLabel.attr("data-src", "./img/" + fileName);
                appendHtml += aLabel.outerHtml();
            }
            div.append(appendHtml);
            bw.write(doc.outerHtml());
            bw.close();
        } catch (Exception e)
        {
            e.printStackTrace();
        }
    }
    @Test
    public void getFileName() {
        File directory = new File("C:\\Users\\Administrator\\Desktop\\beauty\\img");
        File[] pictures = directory.listFiles();
        BufferedImage src = null;
        InputStream is = null;
        int width = -1;
        int height = -1;
        for (File jpg : pictures) {
            try {
                is = new FileInputStream(jpg);
                src = javax.imageio.ImageIO.read(is);
                width = src.getWidth(); // 得到源图宽
                height = src.getHeight();
            } catch (Exception e) {
                e.printStackTrace();
            }
            if ((width / height) < 1d) {
                jpgName.add(jpg.getName());
            }
        }
    }


    @Test
    public void test() {
        String goalUrl = "http://www.zjito.com/upload/resources/image/2016/6/5/75e7a1cb-a76d-4b0c-a620-057883c7f19b_720x1500.jpg?1483471626000";
        System.out.println(goalUrl.substring(goalUrl.lastIndexOf("/") + 1, goalUrl.lastIndexOf(".")));
    }
}
